<!DOCTYPE HTML>
<html lang="en" >
    <!-- Start book Python爬虫课程讲义 -->
    <head>
        <!-- head:start -->
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <title>Spiders | Python爬虫课程讲义</title>
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <meta name="description" content="">
        <meta name="generator" content="GitBook 2.6.7">
        <meta name="author" content="BigCat">
        
        <meta name="HandheldFriendly" content="true"/>
        <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
        <meta name="apple-mobile-web-app-capable" content="yes">
        <meta name="apple-mobile-web-app-status-bar-style" content="black">
        <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
        <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
        
    <link rel="stylesheet" href="../../gitbook/style.css">
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-tbfed-pagefooter/footer.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-splitter/splitter.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-highlight/website.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-fontsettings/website.css">
        
    
    

        
    
    
    <link rel="next" href="../../file/part04/4.6.html" />
    
    
    <link rel="prev" href="../../file/part04/4.4.html" />
    

        <!-- head:end -->
    </head>
    <body>
        <!-- body:start -->
        
    <div class="book"
        data-level="4.5"
        data-chapter-title="Spiders"
        data-filepath="file/part04/4.5.md"
        data-basepath="../.."
        data-revision="Thu Feb 09 2017 09:48:59 GMT+0800 (CST)"
        data-innerlanguage="">
    

<div class="book-summary">
    <nav role="navigation">
        <ul class="summary">
            
            
            
            

            

            
    
        <li class="chapter " data-level="0" data-path="index.html">
            
                
                    <a href="../../index.html">
                
                        <i class="fa fa-check"></i>
                        
                        传智播客Python学院爬虫课程
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1" data-path="file/part01/1.html">
            
                
                    <a href="../../file/part01/1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.</b>
                        
                        爬虫原理与数据抓取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1" data-path="file/part01/1.1.html">
            
                
                    <a href="../../file/part01/1.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.1.</b>
                        
                        (了解)通用爬虫和聚焦爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="file/part01/1.2.html">
            
                
                    <a href="../../file/part01/1.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.2.</b>
                        
                        (复习)HTTP/HTTPS的请求与响应
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="file/part01/1.3.html">
            
                
                    <a href="../../file/part01/1.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.3.</b>
                        
                        HTTP/HTTPS抓包工具-Fiddler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="file/part01/1.4.html">
            
                
                    <a href="../../file/part01/1.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.4.</b>
                        
                        urllib2模块的基本使用
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.5" data-path="file/part01/1.5.html">
            
                
                    <a href="../../file/part01/1.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.5.</b>
                        
                        urllib2：GET请求和POST请求
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.6" data-path="file/part01/1.6.html">
            
                
                    <a href="../../file/part01/1.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.6.</b>
                        
                        urllib2：Handler处理器和自定义Opener
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.7" data-path="file/part01/1.7.html">
            
                
                    <a href="../../file/part01/1.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.7.</b>
                        
                        urllib2：URLError与HTTPError
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.8" data-path="file/part01/1.8.html">
            
                
                    <a href="../../file/part01/1.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.8.</b>
                        
                        Requests模块
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2" data-path="file/part02/2.html">
            
                
                    <a href="../../file/part02/2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.</b>
                        
                        非结构化数据与结构化数据提取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1" data-path="file/part02/2.1.html">
            
                
                    <a href="../../file/part02/2.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.1.</b>
                        
                        正则表达式re模块
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.2" data-path="file/part02/2.2.html">
            
                
                    <a href="../../file/part02/2.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.2.</b>
                        
                        案例：使用正则表达式的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.3" data-path="file/part02/2.3.html">
            
                
                    <a href="../../file/part02/2.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.3.</b>
                        
                        XPath与lxml类库
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.4" data-path="file/part02/2.4.html">
            
                
                    <a href="../../file/part02/2.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.4.</b>
                        
                        案例：使用XPath的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.5" data-path="file/part02/2.5.html">
            
                
                    <a href="../../file/part02/2.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.5.</b>
                        
                        BeautifulSoup4 解析器
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.6" data-path="file/part02/2.6.html">
            
                
                    <a href="../../file/part02/2.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.6.</b>
                        
                        案例：使用bs4的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.7" data-path="file/part02/2.7.html">
            
                
                    <a href="../../file/part02/2.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.7.</b>
                        
                        JSON模块与JsonPath
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.8" data-path="file/part02/2.8.html">
            
                
                    <a href="../../file/part02/2.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.8.</b>
                        
                        糗事百科案例
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.9" data-path="file/part02/2.9.html">
            
                
                    <a href="../../file/part02/2.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.9.</b>
                        
                        多线程爬虫案例
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="3" data-path="file/part03/3.html">
            
                
                    <a href="../../file/part03/3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.</b>
                        
                        动态HTML处理和机器图像识别
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="3.1" data-path="file/part03/3.1.html">
            
                
                    <a href="../../file/part03/3.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.1.</b>
                        
                        动态HTML介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.2" data-path="file/part03/3.2.html">
            
                
                    <a href="../../file/part03/3.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.2.</b>
                        
                        Selenium与PhantomJS
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.3" data-path="file/part03/3.3.html">
            
                
                    <a href="../../file/part03/3.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.3.</b>
                        
                        案例一：网站模拟登录
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.4" data-path="file/part03/3.4.html">
            
                
                    <a href="../../file/part03/3.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.4.</b>
                        
                        案例二：动态页面模拟点击
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.5" data-path="file/part03/3.5.html">
            
                
                    <a href="../../file/part03/3.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.5.</b>
                        
                        案例三：执行JavaScript语句
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.6" data-path="file/part03/3.6.html">
            
                
                    <a href="../../file/part03/3.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.6.</b>
                        
                        机器视觉与Tesseract介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.7" data-path="file/part03/3.7.html">
            
                
                    <a href="../../file/part03/3.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.7.</b>
                        
                        处理一些格式规范的文字
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.8" data-path="file/part03/3.8.html">
            
                
                    <a href="../../file/part03/3.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.8.</b>
                        
                        案例：尝试对验证码进行机器识别处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.9" data-path="file/part03/3.9.html">
            
                
                    <a href="../../file/part03/3.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.9.</b>
                        
                        机器学习：训练Tesseract
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="4" data-path="file/part04/4.html">
            
                
                    <a href="../../file/part04/4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.</b>
                        
                        Scrapy框架
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="4.1" data-path="file/part04/4.1.html">
            
                
                    <a href="../../file/part04/4.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.1.</b>
                        
                        配置安装
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.2" data-path="file/part04/4.2.html">
            
                
                    <a href="../../file/part04/4.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.2.</b>
                        
                        入门案例
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.3" data-path="file/part04/4.3.html">
            
                
                    <a href="../../file/part04/4.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.3.</b>
                        
                        Scrapy Shell
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.4" data-path="file/part04/4.4.html">
            
                
                    <a href="../../file/part04/4.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.4.</b>
                        
                        Item Pipeline
                    </a>
            
            
        </li>
    
        <li class="chapter active" data-level="4.5" data-path="file/part04/4.5.html">
            
                
                    <a href="../../file/part04/4.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.5.</b>
                        
                        Spiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.6" data-path="file/part04/4.6.html">
            
                
                    <a href="../../file/part04/4.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.6.</b>
                        
                        CrawlSpiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.7" data-path="file/part04/4.7.html">
            
                
                    <a href="../../file/part04/4.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.7.</b>
                        
                        Request/Response
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.8" data-path="file/part04/4.8.html">
            
                
                    <a href="../../file/part04/4.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.8.</b>
                        
                        Downloader Middlewares
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.9" data-path="file/part04/4.9.html">
            
                
                    <a href="../../file/part04/4.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.9.</b>
                        
                        Settings
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="5" data-path="file/part05/5.html">
            
                
                    <a href="../../file/part05/5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.</b>
                        
                        Scrapy实战项目
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="5.1" data-path="file/part05/5.1.html">
            
                
                    <a href="../../file/part05/5.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.1.</b>
                        
                        (案例一)手机App抓包爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.2" data-path="file/part05/5.2.html">
            
                
                    <a href="../../file/part05/5.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.2.</b>
                        
                        (案例二)阳光热线问政平台爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.3" data-path="file/part05/5.3.html">
            
                
                    <a href="../../file/part05/5.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.3.</b>
                        
                        (案例三)新浪网分类资讯爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.4" data-path="file/part05/5.4.html">
            
                
                    <a href="../../file/part05/5.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.4.</b>
                        
                        (案例四)图片下载器爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.5" data-path="file/part05/5.5.html">
            
                
                    <a href="../../file/part05/5.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.5.</b>
                        
                        (案例五)将数据保存在MongoDB中
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.6" data-path="file/part05/5.6.html">
            
                
                    <a href="../../file/part05/5.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.6.</b>
                        
                        (案例六)三种scrapy模拟登陆策略
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.7" data-path="file/part05/5.7.html">
            
                
                    <a href="../../file/part05/5.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.7.</b>
                        
                        附：通过Fiddler进行手机抓包方法
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="6" data-path="file/part06/6.html">
            
                
                    <a href="../../file/part06/6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.</b>
                        
                        scrapy-redis分布式组件
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="6.1" data-path="file/part06/6.1.html">
            
                
                    <a href="../../file/part06/6.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.1.</b>
                        
                        源码分析参考：Connection
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.2" data-path="file/part06/6.2.html">
            
                
                    <a href="../../file/part06/6.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.2.</b>
                        
                        源码分析参考：Dupefilter
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.3" data-path="file/part06/6.3.html">
            
                
                    <a href="../../file/part06/6.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.3.</b>
                        
                        源码分析参考：Picklecompat
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.4" data-path="file/part06/6.4.html">
            
                
                    <a href="../../file/part06/6.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.4.</b>
                        
                        源码分析参考：Pipelines
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.5" data-path="file/part06/6.5.html">
            
                
                    <a href="../../file/part06/6.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.5.</b>
                        
                        源码分析参考：Queue
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.6" data-path="file/part06/6.6.html">
            
                
                    <a href="../../file/part06/6.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.6.</b>
                        
                        源码分析参考：Scheduler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.7" data-path="file/part06/6.7.html">
            
                
                    <a href="../../file/part06/6.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.7.</b>
                        
                        源码分析参考：Spider
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="7" data-path="file/part07/7.html">
            
                
                    <a href="../../file/part07/7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.</b>
                        
                        scrapy-redis实战
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="7.1" data-path="file/part07/7.1.html">
            
                
                    <a href="../../file/part07/7.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.1.</b>
                        
                        源码自带项目说明
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.2" data-path="file/part07/7.2.html">
            
                
                    <a href="../../file/part07/7.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.2.</b>
                        
                        有缘网分布式爬虫项目1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.3" data-path="file/part07/7.3.html">
            
                
                    <a href="../../file/part07/7.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.3.</b>
                        
                        有缘网分布式爬虫项目2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.4" data-path="file/part07/7.4.html">
            
                
                    <a href="../../file/part07/7.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.4.</b>
                        
                        处理Redis里的数据
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.5" data-path="file/part07/7.5.html">
            
                
                    <a href="../../file/part07/7.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.5.</b>
                        
                        尝试改写新浪网分类资讯爬虫1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.6" data-path="file/part07/7.6.html">
            
                
                    <a href="../../file/part07/7.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.6.</b>
                        
                        尝试改写新浪网分类资讯爬虫2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.7" data-path="file/part07/7.7.html">
            
                
                    <a href="../../file/part07/7.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.7.</b>
                        
                        IT桔子分布式项目1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.8" data-path="file/part07/7.8.html">
            
                
                    <a href="../../file/part07/7.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.8.</b>
                        
                        IT桔子分布式项目2
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="8" data-path="file/duanzi/duanzi.html">
            
                
                    <a href="../../file/duanzi/duanzi.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>8.</b>
                        
                        课余段子
                    </a>
            
            
        </li>
    


            
            <li class="divider"></li>
            <li>
                <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
                    Published with GitBook
                </a>
            </li>
            
        </ul>
    </nav>
</div>

    <div class="book-body">
        <div class="body-inner">
            <div class="book-header" role="navigation">
    <!-- Actions Left -->
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="../../" >Python爬虫课程讲义</a>
    </h1>
</div>

            <div class="page-wrapper" tabindex="-1" role="main">
                <div class="page-inner">
                
                
                    <section class="normal" id="section-">
                    
                        <h1 id="spider">Spider</h1>
<p>Spider&#x7C7B;&#x5B9A;&#x4E49;&#x4E86;&#x5982;&#x4F55;&#x722C;&#x53D6;&#x67D0;&#x4E2A;(&#x6216;&#x67D0;&#x4E9B;)&#x7F51;&#x7AD9;&#x3002;&#x5305;&#x62EC;&#x4E86;&#x722C;&#x53D6;&#x7684;&#x52A8;&#x4F5C;(&#x4F8B;&#x5982;:&#x662F;&#x5426;&#x8DDF;&#x8FDB;&#x94FE;&#x63A5;)&#x4EE5;&#x53CA;&#x5982;&#x4F55;&#x4ECE;&#x7F51;&#x9875;&#x7684;&#x5185;&#x5BB9;&#x4E2D;&#x63D0;&#x53D6;&#x7ED3;&#x6784;&#x5316;&#x6570;&#x636E;(&#x722C;&#x53D6;item)&#x3002; &#x6362;&#x53E5;&#x8BDD;&#x8BF4;&#xFF0C;Spider&#x5C31;&#x662F;&#x60A8;&#x5B9A;&#x4E49;&#x722C;&#x53D6;&#x7684;&#x52A8;&#x4F5C;&#x53CA;&#x5206;&#x6790;&#x67D0;&#x4E2A;&#x7F51;&#x9875;(&#x6216;&#x8005;&#x662F;&#x6709;&#x4E9B;&#x7F51;&#x9875;)&#x7684;&#x5730;&#x65B9;&#x3002;</p>
<p><code>class scrapy.Spider</code>&#x662F;&#x6700;&#x57FA;&#x672C;&#x7684;&#x7C7B;&#xFF0C;&#x6240;&#x6709;&#x7F16;&#x5199;&#x7684;&#x722C;&#x866B;&#x5FC5;&#x987B;&#x7EE7;&#x627F;&#x8FD9;&#x4E2A;&#x7C7B;&#x3002;</p>
<p>&#x4E3B;&#x8981;&#x7528;&#x5230;&#x7684;&#x51FD;&#x6570;&#x53CA;&#x8C03;&#x7528;&#x987A;&#x5E8F;&#x4E3A;&#xFF1A;</p>
<p><code>__init__()</code> : &#x521D;&#x59CB;&#x5316;&#x722C;&#x866B;&#x540D;&#x5B57;&#x548C;start_urls&#x5217;&#x8868;</p>
<p><code>start_requests() &#x8C03;&#x7528;make_requests_from url()</code>:&#x751F;&#x6210;Requests&#x5BF9;&#x8C61;&#x4EA4;&#x7ED9;Scrapy&#x4E0B;&#x8F7D;&#x5E76;&#x8FD4;&#x56DE;response</p>
<p><code>parse()</code> : &#x89E3;&#x6790;response&#xFF0C;&#x5E76;&#x8FD4;&#x56DE;Item&#x6216;Requests&#xFF08;&#x9700;&#x6307;&#x5B9A;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#xFF09;&#x3002;Item&#x4F20;&#x7ED9;Item pipline&#x6301;&#x4E45;&#x5316; &#xFF0C; &#x800C;Requests&#x4EA4;&#x7531;Scrapy&#x4E0B;&#x8F7D;&#xFF0C;&#x5E76;&#x7531;&#x6307;&#x5B9A;&#x7684;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#x5904;&#x7406;&#xFF08;&#x9ED8;&#x8BA4;parse())&#xFF0C;&#x4E00;&#x76F4;&#x8FDB;&#x884C;&#x5FAA;&#x73AF;&#xFF0C;&#x76F4;&#x5230;&#x5904;&#x7406;&#x5B8C;&#x6240;&#x6709;&#x7684;&#x6570;&#x636E;&#x4E3A;&#x6B62;&#x3002;</p>
<h4 id="&#x6E90;&#x7801;&#x53C2;&#x8003;">&#x6E90;&#x7801;&#x53C2;&#x8003;</h4>
<pre><code class="lang-python"><span class="hljs-comment">#&#x6240;&#x6709;&#x722C;&#x866B;&#x7684;&#x57FA;&#x7C7B;&#xFF0C;&#x7528;&#x6237;&#x5B9A;&#x4E49;&#x7684;&#x722C;&#x866B;&#x5FC5;&#x987B;&#x4ECE;&#x8FD9;&#x4E2A;&#x7C7B;&#x7EE7;&#x627F;</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Spider</span><span class="hljs-params">(object_ref)</span>:</span>

    <span class="hljs-comment">#&#x5B9A;&#x4E49;spider&#x540D;&#x5B57;&#x7684;&#x5B57;&#x7B26;&#x4E32;(string)&#x3002;spider&#x7684;&#x540D;&#x5B57;&#x5B9A;&#x4E49;&#x4E86;Scrapy&#x5982;&#x4F55;&#x5B9A;&#x4F4D;(&#x5E76;&#x521D;&#x59CB;&#x5316;)spider&#xFF0C;&#x6240;&#x4EE5;&#x5176;&#x5FC5;&#x987B;&#x662F;&#x552F;&#x4E00;&#x7684;&#x3002;</span>
    <span class="hljs-comment">#name&#x662F;spider&#x6700;&#x91CD;&#x8981;&#x7684;&#x5C5E;&#x6027;&#xFF0C;&#x800C;&#x4E14;&#x662F;&#x5FC5;&#x987B;&#x7684;&#x3002;</span>
    <span class="hljs-comment">#&#x4E00;&#x822C;&#x505A;&#x6CD5;&#x662F;&#x4EE5;&#x8BE5;&#x7F51;&#x7AD9;(domain)(&#x52A0;&#x6216;&#x4E0D;&#x52A0; &#x540E;&#x7F00; )&#x6765;&#x547D;&#x540D;spider&#x3002; &#x4F8B;&#x5982;&#xFF0C;&#x5982;&#x679C;spider&#x722C;&#x53D6; mywebsite.com &#xFF0C;&#x8BE5;spider&#x901A;&#x5E38;&#x4F1A;&#x88AB;&#x547D;&#x540D;&#x4E3A; mywebsite</span>
    name = <span class="hljs-keyword">None</span>

    <span class="hljs-comment">#&#x521D;&#x59CB;&#x5316;&#xFF0C;&#x63D0;&#x53D6;&#x722C;&#x866B;&#x540D;&#x5B57;&#xFF0C;start_ruls</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, name=None, **kwargs)</span>:</span>
        <span class="hljs-keyword">if</span> name <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">None</span>:
            self.name = name
        <span class="hljs-comment"># &#x5982;&#x679C;&#x722C;&#x866B;&#x6CA1;&#x6709;&#x540D;&#x5B57;&#xFF0C;&#x4E2D;&#x65AD;&#x540E;&#x7EED;&#x64CD;&#x4F5C;&#x5219;&#x62A5;&#x9519;</span>
        <span class="hljs-keyword">elif</span> <span class="hljs-keyword">not</span> getattr(self, <span class="hljs-string">&apos;name&apos;</span>, <span class="hljs-keyword">None</span>):
            <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">&quot;%s must have a name&quot;</span> % type(self).__name__)

        <span class="hljs-comment"># python &#x5BF9;&#x8C61;&#x6216;&#x7C7B;&#x578B;&#x901A;&#x8FC7;&#x5185;&#x7F6E;&#x6210;&#x5458;__dict__&#x6765;&#x5B58;&#x50A8;&#x6210;&#x5458;&#x4FE1;&#x606F;</span>
        self.__dict__.update(kwargs)

        <span class="hljs-comment">#URL&#x5217;&#x8868;&#x3002;&#x5F53;&#x6CA1;&#x6709;&#x6307;&#x5B9A;&#x7684;URL&#x65F6;&#xFF0C;spider&#x5C06;&#x4ECE;&#x8BE5;&#x5217;&#x8868;&#x4E2D;&#x5F00;&#x59CB;&#x8FDB;&#x884C;&#x722C;&#x53D6;&#x3002; &#x56E0;&#x6B64;&#xFF0C;&#x7B2C;&#x4E00;&#x4E2A;&#x88AB;&#x83B7;&#x53D6;&#x5230;&#x7684;&#x9875;&#x9762;&#x7684;URL&#x5C06;&#x662F;&#x8BE5;&#x5217;&#x8868;&#x4E4B;&#x4E00;&#x3002; &#x540E;&#x7EED;&#x7684;URL&#x5C06;&#x4F1A;&#x4ECE;&#x83B7;&#x53D6;&#x5230;&#x7684;&#x6570;&#x636E;&#x4E2D;&#x63D0;&#x53D6;&#x3002;</span>
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> hasattr(self, <span class="hljs-string">&apos;start_urls&apos;</span>):
            self.start_urls = []

    <span class="hljs-comment"># &#x6253;&#x5370;Scrapy&#x6267;&#x884C;&#x540E;&#x7684;log&#x4FE1;&#x606F;</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">log</span><span class="hljs-params">(self, message, level=log.DEBUG, **kw)</span>:</span>
        log.msg(message, spider=self, level=level, **kw)

    <span class="hljs-comment"># &#x5224;&#x65AD;&#x5BF9;&#x8C61;object&#x7684;&#x5C5E;&#x6027;&#x662F;&#x5426;&#x5B58;&#x5728;&#xFF0C;&#x4E0D;&#x5B58;&#x5728;&#x505A;&#x65AD;&#x8A00;&#x5904;&#x7406;</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">set_crawler</span><span class="hljs-params">(self, crawler)</span>:</span>
        <span class="hljs-keyword">assert</span> <span class="hljs-keyword">not</span> hasattr(self, <span class="hljs-string">&apos;_crawler&apos;</span>), <span class="hljs-string">&quot;Spider already bounded to %s&quot;</span> % crawler
        self._crawler = crawler

<span class="hljs-decorator">    @property</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">crawler</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">assert</span> hasattr(self, <span class="hljs-string">&apos;_crawler&apos;</span>), <span class="hljs-string">&quot;Spider not bounded to any crawler&quot;</span>
        <span class="hljs-keyword">return</span> self._crawler

<span class="hljs-decorator">    @property</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">settings</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">return</span> self.crawler.settings

    <span class="hljs-comment">#&#x8BE5;&#x65B9;&#x6CD5;&#x5C06;&#x8BFB;&#x53D6;start_urls&#x5185;&#x7684;&#x5730;&#x5740;&#xFF0C;&#x5E76;&#x4E3A;&#x6BCF;&#x4E00;&#x4E2A;&#x5730;&#x5740;&#x751F;&#x6210;&#x4E00;&#x4E2A;Request&#x5BF9;&#x8C61;&#xFF0C;&#x4EA4;&#x7ED9;Scrapy&#x4E0B;&#x8F7D;&#x5E76;&#x8FD4;&#x56DE;Response</span>
    <span class="hljs-comment">#&#x8BE5;&#x65B9;&#x6CD5;&#x4EC5;&#x8C03;&#x7528;&#x4E00;&#x6B21;</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">start_requests</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> self.start_urls:
            <span class="hljs-keyword">yield</span> self.make_requests_from_url(url)

    <span class="hljs-comment">#start_requests()&#x4E2D;&#x8C03;&#x7528;&#xFF0C;&#x5B9E;&#x9645;&#x751F;&#x6210;Request&#x7684;&#x51FD;&#x6570;&#x3002;</span>
    <span class="hljs-comment">#Request&#x5BF9;&#x8C61;&#x9ED8;&#x8BA4;&#x7684;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#x4E3A;parse()&#xFF0C;&#x63D0;&#x4EA4;&#x7684;&#x65B9;&#x5F0F;&#x4E3A;get</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">make_requests_from_url</span><span class="hljs-params">(self, url)</span>:</span>
        <span class="hljs-keyword">return</span> Request(url, dont_filter=<span class="hljs-keyword">True</span>)

    <span class="hljs-comment">#&#x9ED8;&#x8BA4;&#x7684;Request&#x5BF9;&#x8C61;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#xFF0C;&#x5904;&#x7406;&#x8FD4;&#x56DE;&#x7684;response&#x3002;</span>
    <span class="hljs-comment">#&#x751F;&#x6210;Item&#x6216;&#x8005;Request&#x5BF9;&#x8C61;&#x3002;&#x7528;&#x6237;&#x5FC5;&#x987B;&#x5B9E;&#x73B0;&#x8FD9;&#x4E2A;&#x7C7B;</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse</span><span class="hljs-params">(self, response)</span>:</span>
        <span class="hljs-keyword">raise</span> NotImplementedError

<span class="hljs-decorator">    @classmethod</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">handles_request</span><span class="hljs-params">(cls, request)</span>:</span>
        <span class="hljs-keyword">return</span> url_is_from_spider(request.url, cls)

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__str__</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-keyword">return</span> <span class="hljs-string">&quot;&lt;%s %r at 0x%0x&gt;&quot;</span> % (type(self).__name__, self.name, id(self))

    __repr__ = __str__
</code></pre>
<h4 id="&#x4E3B;&#x8981;&#x5C5E;&#x6027;&#x548C;&#x65B9;&#x6CD5;">&#x4E3B;&#x8981;&#x5C5E;&#x6027;&#x548C;&#x65B9;&#x6CD5;</h4>
<ul>
<li><p>name</p>
<blockquote>
<p>&#x5B9A;&#x4E49;spider&#x540D;&#x5B57;&#x7684;&#x5B57;&#x7B26;&#x4E32;&#x3002;</p>
<p>&#x4F8B;&#x5982;&#xFF0C;&#x5982;&#x679C;spider&#x722C;&#x53D6; mywebsite.com &#xFF0C;&#x8BE5;spider&#x901A;&#x5E38;&#x4F1A;&#x88AB;&#x547D;&#x540D;&#x4E3A; mywebsite</p>
</blockquote>
</li>
<li><p>allowed_domains</p>
<blockquote>
<p>&#x5305;&#x542B;&#x4E86;spider&#x5141;&#x8BB8;&#x722C;&#x53D6;&#x7684;&#x57DF;&#x540D;(domain)&#x7684;&#x5217;&#x8868;&#xFF0C;&#x53EF;&#x9009;&#x3002;</p>
</blockquote>
</li>
<li><p>start_urls</p>
<blockquote>
<p>&#x521D;&#x59CB;URL&#x5143;&#x7956;/&#x5217;&#x8868;&#x3002;&#x5F53;&#x6CA1;&#x6709;&#x5236;&#x5B9A;&#x7279;&#x5B9A;&#x7684;URL&#x65F6;&#xFF0C;spider&#x5C06;&#x4ECE;&#x8BE5;&#x5217;&#x8868;&#x4E2D;&#x5F00;&#x59CB;&#x8FDB;&#x884C;&#x722C;&#x53D6;&#x3002;</p>
</blockquote>
</li>
<li><p>start_requests(self)</p>
<blockquote>
<p>&#x8BE5;&#x65B9;&#x6CD5;&#x5FC5;&#x987B;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x53EF;&#x8FED;&#x4EE3;&#x5BF9;&#x8C61;(iterable)&#x3002;&#x8BE5;&#x5BF9;&#x8C61;&#x5305;&#x542B;&#x4E86;spider&#x7528;&#x4E8E;&#x722C;&#x53D6;&#xFF08;&#x9ED8;&#x8BA4;&#x5B9E;&#x73B0;&#x662F;&#x4F7F;&#x7528; start_urls &#x7684;url&#xFF09;&#x7684;&#x7B2C;&#x4E00;&#x4E2A;Request&#x3002;</p>
<p>&#x5F53;spider&#x542F;&#x52A8;&#x722C;&#x53D6;&#x5E76;&#x4E14;&#x672A;&#x6307;&#x5B9A;start_urls&#x65F6;&#xFF0C;&#x8BE5;&#x65B9;&#x6CD5;&#x88AB;&#x8C03;&#x7528;&#x3002;</p>
</blockquote>
</li>
<li><p>parse(self, response)</p>
<blockquote>
<p>&#x5F53;&#x8BF7;&#x6C42;url&#x8FD4;&#x56DE;&#x7F51;&#x9875;&#x6CA1;&#x6709;&#x6307;&#x5B9A;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#x65F6;&#xFF0C;&#x9ED8;&#x8BA4;&#x7684;Request&#x5BF9;&#x8C61;&#x56DE;&#x8C03;&#x51FD;&#x6570;&#x3002;&#x7528;&#x6765;&#x5904;&#x7406;&#x7F51;&#x9875;&#x8FD4;&#x56DE;&#x7684;response&#xFF0C;&#x4EE5;&#x53CA;&#x751F;&#x6210;Item&#x6216;&#x8005;Request&#x5BF9;&#x8C61;&#x3002;</p>
</blockquote>
</li>
<li><p>log(self, message[, level, component])</p>
<blockquote>
<p>&#x4F7F;&#x7528; scrapy.log.msg() &#x65B9;&#x6CD5;&#x8BB0;&#x5F55;(log)message&#x3002; &#x66F4;&#x591A;&#x6570;&#x636E;&#x8BF7;&#x53C2;&#x89C1; <a href="4.7.html">logging</a></p>
</blockquote>
</li>
</ul>
<h4 id="&#x6848;&#x4F8B;&#xFF1A;&#x817E;&#x8BAF;&#x62DB;&#x8058;&#x7F51;&#x81EA;&#x52A8;&#x7FFB;&#x9875;&#x91C7;&#x96C6;">&#x6848;&#x4F8B;&#xFF1A;&#x817E;&#x8BAF;&#x62DB;&#x8058;&#x7F51;&#x81EA;&#x52A8;&#x7FFB;&#x9875;&#x91C7;&#x96C6;</h4>
<ul>
<li>&#x521B;&#x5EFA;&#x4E00;&#x4E2A;&#x65B0;&#x7684;&#x722C;&#x866B;&#xFF1A; </li>
</ul>
<p><code>scrapy genspider tencent &quot;tencent.com&quot;</code></p>
<ul>
<li>&#x7F16;&#x5199;items.py</li>
</ul>
<p>&#x83B7;&#x53D6;&#x804C;&#x4F4D;&#x540D;&#x79F0;&#x3001;&#x8BE6;&#x7EC6;&#x4FE1;&#x606F;&#x3001;</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">TencentItem</span><span class="hljs-params">(scrapy.Item)</span>:</span>
    name = scrapy.Field()
    detailLink = scrapy.Field()
    positionInfo = scrapy.Field()
    peopleNumber = scrapy.Field()
    workLocation = scrapy.Field()
    publishTime = scrapy.Field()
</code></pre>
<ul>
<li>&#x7F16;&#x5199;tencent.py</li>
</ul>
<pre><code class="lang-python"><span class="hljs-comment"># tencent.py</span>

<span class="hljs-keyword">from</span> mySpider.items <span class="hljs-keyword">import</span> TencentItem
<span class="hljs-keyword">import</span> scrapy
<span class="hljs-keyword">import</span> re

<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">TencentSpider</span><span class="hljs-params">(scrapy.Spider)</span>:</span>
    name = <span class="hljs-string">&quot;tencent&quot;</span>
    allowed_domains = [<span class="hljs-string">&quot;hr.tencent.com&quot;</span>]
    start_urls = [
        <span class="hljs-string">&quot;http://hr.tencent.com/position.php?&amp;start=0#a&quot;</span>
    ]

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse</span><span class="hljs-params">(self, response)</span>:</span>
        <span class="hljs-keyword">for</span> each <span class="hljs-keyword">in</span> response.xpath(<span class="hljs-string">&apos;//*[@class=&quot;even&quot;]&apos;</span>):

            item = TencentItem()
            name = each.xpath(<span class="hljs-string">&apos;./td[1]/a/text()&apos;</span>).extract()[<span class="hljs-number">0</span>]
            detailLink = each.xpath(<span class="hljs-string">&apos;./td[1]/a/@href&apos;</span>).extract()[<span class="hljs-number">0</span>]
            positionInfo = each.xpath(<span class="hljs-string">&apos;./td[2]/text()&apos;</span>).extract()[<span class="hljs-number">0</span>]
            peopleNumber = each.xpath(<span class="hljs-string">&apos;./td[3]/text()&apos;</span>).extract()[<span class="hljs-number">0</span>]
            workLocation = each.xpath(<span class="hljs-string">&apos;./td[4]/text()&apos;</span>).extract()[<span class="hljs-number">0</span>]
            publishTime = each.xpath(<span class="hljs-string">&apos;./td[5]/text()&apos;</span>).extract()[<span class="hljs-number">0</span>]

            <span class="hljs-comment">#print name, detailLink, catalog, peopleNumber, workLocation,publishTime</span>

            item[<span class="hljs-string">&apos;name&apos;</span>] = name.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)
            item[<span class="hljs-string">&apos;detailLink&apos;</span>] = detailLink.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)
            item[<span class="hljs-string">&apos;positionInfo&apos;</span>] = positionInfo.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)
            item[<span class="hljs-string">&apos;peopleNumber&apos;</span>] = peopleNumber.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)
            item[<span class="hljs-string">&apos;workLocation&apos;</span>] = workLocation.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)
            item[<span class="hljs-string">&apos;publishTime&apos;</span>] = publishTime.encode(<span class="hljs-string">&apos;utf-8&apos;</span>)

            curpage = re.search(<span class="hljs-string">&apos;(\d+)&apos;</span>,response.url).group(<span class="hljs-number">1</span>)
            page = int(curpage) + <span class="hljs-number">10</span>
            url = re.sub(<span class="hljs-string">&apos;\d+&apos;</span>, str(page), response.url)

            <span class="hljs-comment"># &#x53D1;&#x9001;&#x65B0;&#x7684;url&#x8BF7;&#x6C42;&#x52A0;&#x5165;&#x5F85;&#x722C;&#x961F;&#x5217;&#xFF0C;&#x5E76;&#x8C03;&#x7528;&#x56DE;&#x8C03;&#x51FD;&#x6570; self.parse</span>
            <span class="hljs-keyword">yield</span> scrapy.Request(url, callback = self.parse)

            <span class="hljs-comment"># &#x5C06;&#x83B7;&#x53D6;&#x7684;&#x6570;&#x636E;&#x4EA4;&#x7ED9;pipeline</span>
            <span class="hljs-keyword">yield</span> item
</code></pre>
<ul>
<li>&#x7F16;&#x5199;pipeline.py&#x6587;&#x4EF6;</li>
</ul>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> json

<span class="hljs-comment">#class ItcastJsonPipeline(object):</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">TencentJsonPipeline</span><span class="hljs-params">(object)</span>:</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-comment">#self.file = open(&apos;teacher.json&apos;, &apos;wb&apos;)</span>
        self.file = open(<span class="hljs-string">&apos;tencent.json&apos;</span>, <span class="hljs-string">&apos;wb&apos;</span>)

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_item</span><span class="hljs-params">(self, item, spider)</span>:</span>
        content = json.dumps(dict(item), ensure_ascii=<span class="hljs-keyword">False</span>) + <span class="hljs-string">&quot;\n&quot;</span>
        self.file.write(content)
        <span class="hljs-keyword">return</span> item

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">close_spider</span><span class="hljs-params">(self, spider)</span>:</span>
        self.file.close()
</code></pre>
<ul>
<li>&#x5728; setting.py &#x91CC;&#x8BBE;&#x7F6E;ITEM_PIPELINES</li>
</ul>
<pre><code class="lang-python">ITEM_PIPELINES = {
    <span class="hljs-comment">#&apos;mySpider.pipelines.SomePipeline&apos;: 300,</span>
    <span class="hljs-comment">#&quot;mySpider.pipelines.ItcastJsonPipeline&quot;:300</span>
    <span class="hljs-string">&quot;mySpider.pipelines.TencentJsonPipeline&quot;</span>:<span class="hljs-number">300</span>
}
</code></pre>
<ul>
<li>&#x6267;&#x884C;&#x722C;&#x866B;&#xFF1A;<code>scrapy crawl tencent</code></li>
</ul>
<h2 id="&#x601D;&#x8003;">&#x601D;&#x8003;</h2>
<h4 id="&#x8BF7;&#x601D;&#x8003;-parse&#x65B9;&#x6CD5;&#x7684;&#x5DE5;&#x4F5C;&#x673A;&#x5236;&#xFF1A;">&#x8BF7;&#x601D;&#x8003; parse()&#x65B9;&#x6CD5;&#x7684;&#x5DE5;&#x4F5C;&#x673A;&#x5236;&#xFF1A;</h4>
<pre><code class="lang-python">
<span class="hljs-number">1.</span> &#x56E0;&#x4E3A;&#x4F7F;&#x7528;&#x7684;<span class="hljs-keyword">yield</span>&#xFF0C;&#x800C;&#x4E0D;&#x662F;<span class="hljs-keyword">return</span>&#x3002;parse&#x51FD;&#x6570;&#x5C06;&#x4F1A;&#x88AB;&#x5F53;&#x505A;&#x4E00;&#x4E2A;&#x751F;&#x6210;&#x5668;&#x4F7F;&#x7528;&#x3002;scrapy&#x4F1A;&#x9010;&#x4E00;&#x83B7;&#x53D6;parse&#x65B9;&#x6CD5;&#x4E2D;&#x751F;&#x6210;&#x7684;&#x7ED3;&#x679C;&#xFF0C;&#x5E76;&#x5224;&#x65AD;&#x8BE5;&#x7ED3;&#x679C;&#x662F;&#x4E00;&#x4E2A;&#x4EC0;&#x4E48;&#x6837;&#x7684;&#x7C7B;&#x578B;&#xFF1B;
<span class="hljs-number">2.</span> &#x5982;&#x679C;&#x662F;request&#x5219;&#x52A0;&#x5165;&#x722C;&#x53D6;&#x961F;&#x5217;&#xFF0C;&#x5982;&#x679C;&#x662F;item&#x7C7B;&#x578B;&#x5219;&#x4F7F;&#x7528;pipeline&#x5904;&#x7406;&#xFF0C;&#x5176;&#x4ED6;&#x7C7B;&#x578B;&#x5219;&#x8FD4;&#x56DE;&#x9519;&#x8BEF;&#x4FE1;&#x606F;&#x3002;
<span class="hljs-number">3.</span> scrapy&#x53D6;&#x5230;&#x7B2C;&#x4E00;&#x90E8;&#x5206;&#x7684;request&#x4E0D;&#x4F1A;&#x7ACB;&#x9A6C;&#x5C31;&#x53BB;&#x53D1;&#x9001;&#x8FD9;&#x4E2A;request&#xFF0C;&#x53EA;&#x662F;&#x628A;&#x8FD9;&#x4E2A;request&#x653E;&#x5230;&#x961F;&#x5217;&#x91CC;&#xFF0C;&#x7136;&#x540E;&#x63A5;&#x7740;&#x4ECE;&#x751F;&#x6210;&#x5668;&#x91CC;&#x83B7;&#x53D6;&#xFF1B;
<span class="hljs-number">4.</span> &#x53D6;&#x5C3D;&#x7B2C;&#x4E00;&#x90E8;&#x5206;&#x7684;request&#xFF0C;&#x7136;&#x540E;&#x518D;&#x83B7;&#x53D6;&#x7B2C;&#x4E8C;&#x90E8;&#x5206;&#x7684;item&#xFF0C;&#x53D6;&#x5230;item&#x4E86;&#xFF0C;&#x5C31;&#x4F1A;&#x653E;&#x5230;&#x5BF9;&#x5E94;&#x7684;pipeline&#x91CC;&#x5904;&#x7406;&#xFF1B;
<span class="hljs-number">5.</span> parse()&#x65B9;&#x6CD5;&#x4F5C;&#x4E3A;&#x56DE;&#x8C03;&#x51FD;&#x6570;(callback)&#x8D4B;&#x503C;&#x7ED9;&#x4E86;Request&#xFF0C;&#x6307;&#x5B9A;parse()&#x65B9;&#x6CD5;&#x6765;&#x5904;&#x7406;&#x8FD9;&#x4E9B;&#x8BF7;&#x6C42; scrapy.Request(url, callback=self.parse)
<span class="hljs-number">6.</span> Request&#x5BF9;&#x8C61;&#x7ECF;&#x8FC7;&#x8C03;&#x5EA6;&#xFF0C;&#x6267;&#x884C;&#x751F;&#x6210; scrapy.http.response()&#x7684;&#x54CD;&#x5E94;&#x5BF9;&#x8C61;&#xFF0C;&#x5E76;&#x9001;&#x56DE;&#x7ED9;parse()&#x65B9;&#x6CD5;&#xFF0C;&#x76F4;&#x5230;&#x8C03;&#x5EA6;&#x5668;&#x4E2D;&#x6CA1;&#x6709;Request&#xFF08;&#x9012;&#x5F52;&#x7684;&#x601D;&#x8DEF;&#xFF09;
<span class="hljs-number">7.</span> &#x53D6;&#x5C3D;&#x4E4B;&#x540E;&#xFF0C;parse()&#x5DE5;&#x4F5C;&#x7ED3;&#x675F;&#xFF0C;&#x5F15;&#x64CE;&#x518D;&#x6839;&#x636E;&#x961F;&#x5217;&#x548C;pipelines&#x4E2D;&#x7684;&#x5185;&#x5BB9;&#x53BB;&#x6267;&#x884C;&#x76F8;&#x5E94;&#x7684;&#x64CD;&#x4F5C;&#xFF1B;
<span class="hljs-number">8.</span> &#x7A0B;&#x5E8F;&#x5728;&#x53D6;&#x5F97;&#x5404;&#x4E2A;&#x9875;&#x9762;&#x7684;items&#x524D;&#xFF0C;&#x4F1A;&#x5148;&#x5904;&#x7406;&#x5B8C;&#x4E4B;&#x524D;&#x6240;&#x6709;&#x7684;request&#x961F;&#x5217;&#x91CC;&#x7684;&#x8BF7;&#x6C42;&#xFF0C;&#x7136;&#x540E;&#x518D;&#x63D0;&#x53D6;items&#x3002;
<span class="hljs-number">7.</span> &#x8FD9;&#x4E00;&#x5207;&#x7684;&#x4E00;&#x5207;&#xFF0C;Scrapy&#x5F15;&#x64CE;&#x548C;&#x8C03;&#x5EA6;&#x5668;&#x5C06;&#x8D1F;&#x8D23;&#x5230;&#x5E95;&#x3002;
</code></pre>
<footer class="page-footer"><span class="copyright">Copyright &#xA9; BigCat all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x300C;Revision Time:
2016-12-09 03:17:10&#x300D;
</span></footer>
                    
                    </section>
                
                
                </div>
            </div>
        </div>

        
        <a href="../../file/part04/4.4.html" class="navigation navigation-prev " aria-label="Previous page: Item Pipeline"><i class="fa fa-angle-left"></i></a>
        
        
        <a href="../../file/part04/4.6.html" class="navigation navigation-next " aria-label="Next page: CrawlSpiders"><i class="fa fa-angle-right"></i></a>
        
    </div>
</div>

        
<script src="../../gitbook/app.js"></script>

    
    <script src="../../gitbook/plugins/gitbook-plugin-splitter/splitter.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-fontsettings/buttons.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-livereload/plugin.js"></script>
    

<script>
require(["gitbook"], function(gitbook) {
    var config = {"disqus":{"shortName":"gitbookuse"},"github":{"url":"https://github.com/dododream"},"search-pro":{"cutWordLib":"nodejieba","defineWord":["gitbook-use"]},"sharing":{"weibo":true,"facebook":true,"twitter":true,"google":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"tbfed-pagefooter":{"copyright":"Copyright © BigCat","modify_label":"「Revision Time:","modify_format":"YYYY-MM-DD HH:mm:ss」"},"baidu":{"token":"ff100361cdce95dd4c8fb96b4009f7bc"},"sitemap":{"hostname":"http://www.treenewbee.top"},"donate":{"wechat":"http://weixin.png","alipay":"http://alipay.png","title":"","button":"赏","alipayText":"支付宝打赏","wechatText":"微信打赏"},"edit-link":{"base":"https://github.com/dododream/edit","label":"Edit This Page"},"splitter":{},"toggle-chapters":{},"highlight":{},"fontsettings":{"theme":"white","family":"sans","size":2},"livereload":{}};
    gitbook.start(config);
});
</script>

        <!-- body:end -->
    </body>
    <!-- End of book Python爬虫课程讲义 -->
</html>
